{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第二周作业 在Rental Listing Inquiries数据上练习分类方法\n",
    "Rental Listing Inquiries数据集是Kaggle平台上的一个分类竞赛任务，需要根据公寓的特征来预测其受欢迎程度（用户感兴趣程度分为高、中、低三类）。其中房屋的特征x共有14维，响应值y为用户对该公寓的感兴趣程度。评价标准为logloss。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "导入必要的工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import  CountVectorizer\n",
    "from scipy import sparse\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.cluster import KMeans\n",
    "from nltk.metrics import distance as distance\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "加载原始数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_json(\"./data/RentListingInquries_train.json\")\n",
    "test = pd.read_json(\"./data/RentListingInquries_test.json\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "训练集特征数据了解"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>building_id</th>\n",
       "      <th>created</th>\n",
       "      <th>description</th>\n",
       "      <th>display_address</th>\n",
       "      <th>features</th>\n",
       "      <th>interest_level</th>\n",
       "      <th>latitude</th>\n",
       "      <th>listing_id</th>\n",
       "      <th>longitude</th>\n",
       "      <th>manager_id</th>\n",
       "      <th>photos</th>\n",
       "      <th>price</th>\n",
       "      <th>street_address</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>53a5b119ba8f7b61d4e010512e0dfc85</td>\n",
       "      <td>2016-06-24 07:54:24</td>\n",
       "      <td>A Brand New 3 Bedroom 1.5 bath ApartmentEnjoy ...</td>\n",
       "      <td>Metropolitan Avenue</td>\n",
       "      <td>[]</td>\n",
       "      <td>medium</td>\n",
       "      <td>40.7145</td>\n",
       "      <td>7211212</td>\n",
       "      <td>-73.9425</td>\n",
       "      <td>5ba989232d0489da1b5f2c45f6688adc</td>\n",
       "      <td>[https://photos.renthop.com/2/7211212_1ed4542e...</td>\n",
       "      <td>3000</td>\n",
       "      <td>792 Metropolitan Avenue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10000</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>c5c8a357cba207596b04d1afd1e4f130</td>\n",
       "      <td>2016-06-12 12:19:27</td>\n",
       "      <td></td>\n",
       "      <td>Columbus Avenue</td>\n",
       "      <td>[Doorman, Elevator, Fitness Center, Cats Allow...</td>\n",
       "      <td>low</td>\n",
       "      <td>40.7947</td>\n",
       "      <td>7150865</td>\n",
       "      <td>-73.9667</td>\n",
       "      <td>7533621a882f71e25173b27e3139d83d</td>\n",
       "      <td>[https://photos.renthop.com/2/7150865_be3306c5...</td>\n",
       "      <td>5465</td>\n",
       "      <td>808 Columbus Avenue</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100004</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>c3ba40552e2120b0acfc3cb5730bb2aa</td>\n",
       "      <td>2016-04-17 03:26:41</td>\n",
       "      <td>Top Top West Village location, beautiful Pre-w...</td>\n",
       "      <td>W 13 Street</td>\n",
       "      <td>[Laundry In Building, Dishwasher, Hardwood Flo...</td>\n",
       "      <td>high</td>\n",
       "      <td>40.7388</td>\n",
       "      <td>6887163</td>\n",
       "      <td>-74.0018</td>\n",
       "      <td>d9039c43983f6e564b1482b273bd7b01</td>\n",
       "      <td>[https://photos.renthop.com/2/6887163_de85c427...</td>\n",
       "      <td>2850</td>\n",
       "      <td>241 W 13 Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100007</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>28d9ad350afeaab8027513a3e52ac8d5</td>\n",
       "      <td>2016-04-18 02:22:02</td>\n",
       "      <td>Building Amenities - Garage - Garden - fitness...</td>\n",
       "      <td>East 49th Street</td>\n",
       "      <td>[Hardwood Floors, No Fee]</td>\n",
       "      <td>low</td>\n",
       "      <td>40.7539</td>\n",
       "      <td>6888711</td>\n",
       "      <td>-73.9677</td>\n",
       "      <td>1067e078446a7897d2da493d2f741316</td>\n",
       "      <td>[https://photos.renthop.com/2/6888711_6e660cee...</td>\n",
       "      <td>3275</td>\n",
       "      <td>333 East 49th Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100013</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2016-04-28 01:32:41</td>\n",
       "      <td>Beautifully renovated 3 bedroom flex 4 bedroom...</td>\n",
       "      <td>West 143rd Street</td>\n",
       "      <td>[Pre-War]</td>\n",
       "      <td>low</td>\n",
       "      <td>40.8241</td>\n",
       "      <td>6934781</td>\n",
       "      <td>-73.9493</td>\n",
       "      <td>98e13ad4b495b9613cef886d79a6291f</td>\n",
       "      <td>[https://photos.renthop.com/2/6934781_1fa4b41a...</td>\n",
       "      <td>3350</td>\n",
       "      <td>500 West 143rd Street</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        bathrooms  bedrooms                       building_id  \\\n",
       "10            1.5         3  53a5b119ba8f7b61d4e010512e0dfc85   \n",
       "10000         1.0         2  c5c8a357cba207596b04d1afd1e4f130   \n",
       "100004        1.0         1  c3ba40552e2120b0acfc3cb5730bb2aa   \n",
       "100007        1.0         1  28d9ad350afeaab8027513a3e52ac8d5   \n",
       "100013        1.0         4                                 0   \n",
       "\n",
       "                    created  \\\n",
       "10      2016-06-24 07:54:24   \n",
       "10000   2016-06-12 12:19:27   \n",
       "100004  2016-04-17 03:26:41   \n",
       "100007  2016-04-18 02:22:02   \n",
       "100013  2016-04-28 01:32:41   \n",
       "\n",
       "                                              description  \\\n",
       "10      A Brand New 3 Bedroom 1.5 bath ApartmentEnjoy ...   \n",
       "10000                                                       \n",
       "100004  Top Top West Village location, beautiful Pre-w...   \n",
       "100007  Building Amenities - Garage - Garden - fitness...   \n",
       "100013  Beautifully renovated 3 bedroom flex 4 bedroom...   \n",
       "\n",
       "            display_address  \\\n",
       "10      Metropolitan Avenue   \n",
       "10000       Columbus Avenue   \n",
       "100004          W 13 Street   \n",
       "100007     East 49th Street   \n",
       "100013    West 143rd Street   \n",
       "\n",
       "                                                 features interest_level  \\\n",
       "10                                                     []         medium   \n",
       "10000   [Doorman, Elevator, Fitness Center, Cats Allow...            low   \n",
       "100004  [Laundry In Building, Dishwasher, Hardwood Flo...           high   \n",
       "100007                          [Hardwood Floors, No Fee]            low   \n",
       "100013                                          [Pre-War]            low   \n",
       "\n",
       "        latitude  listing_id  longitude                        manager_id  \\\n",
       "10       40.7145     7211212   -73.9425  5ba989232d0489da1b5f2c45f6688adc   \n",
       "10000    40.7947     7150865   -73.9667  7533621a882f71e25173b27e3139d83d   \n",
       "100004   40.7388     6887163   -74.0018  d9039c43983f6e564b1482b273bd7b01   \n",
       "100007   40.7539     6888711   -73.9677  1067e078446a7897d2da493d2f741316   \n",
       "100013   40.8241     6934781   -73.9493  98e13ad4b495b9613cef886d79a6291f   \n",
       "\n",
       "                                                   photos  price  \\\n",
       "10      [https://photos.renthop.com/2/7211212_1ed4542e...   3000   \n",
       "10000   [https://photos.renthop.com/2/7150865_be3306c5...   5465   \n",
       "100004  [https://photos.renthop.com/2/6887163_de85c427...   2850   \n",
       "100007  [https://photos.renthop.com/2/6888711_6e660cee...   3275   \n",
       "100013  [https://photos.renthop.com/2/6934781_1fa4b41a...   3350   \n",
       "\n",
       "                 street_address  \n",
       "10      792 Metropolitan Avenue  \n",
       "10000       808 Columbus Avenue  \n",
       "100004          241 W 13 Street  \n",
       "100007     333 East 49th Street  \n",
       "100013    500 West 143rd Street  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 49352 entries, 10 to 99994\n",
      "Data columns (total 15 columns):\n",
      "bathrooms          49352 non-null float64\n",
      "bedrooms           49352 non-null int64\n",
      "building_id        49352 non-null object\n",
      "created            49352 non-null object\n",
      "description        49352 non-null object\n",
      "display_address    49352 non-null object\n",
      "features           49352 non-null object\n",
      "interest_level     49352 non-null object\n",
      "latitude           49352 non-null float64\n",
      "listing_id         49352 non-null int64\n",
      "longitude          49352 non-null float64\n",
      "manager_id         49352 non-null object\n",
      "photos             49352 non-null object\n",
      "price              49352 non-null int64\n",
      "street_address     49352 non-null object\n",
      "dtypes: float64(3), int64(3), object(9)\n",
      "memory usage: 6.0+ MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>latitude</th>\n",
       "      <th>listing_id</th>\n",
       "      <th>longitude</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>49352.00000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>4.935200e+04</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>4.935200e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.21218</td>\n",
       "      <td>1.541640</td>\n",
       "      <td>40.741545</td>\n",
       "      <td>7.024055e+06</td>\n",
       "      <td>-73.955716</td>\n",
       "      <td>3.830174e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.50142</td>\n",
       "      <td>1.115018</td>\n",
       "      <td>0.638535</td>\n",
       "      <td>1.262746e+05</td>\n",
       "      <td>1.177912</td>\n",
       "      <td>2.206687e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.811957e+06</td>\n",
       "      <td>-118.271000</td>\n",
       "      <td>4.300000e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>40.728300</td>\n",
       "      <td>6.915888e+06</td>\n",
       "      <td>-73.991700</td>\n",
       "      <td>2.500000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>40.751800</td>\n",
       "      <td>7.021070e+06</td>\n",
       "      <td>-73.977900</td>\n",
       "      <td>3.150000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>40.774300</td>\n",
       "      <td>7.128733e+06</td>\n",
       "      <td>-73.954800</td>\n",
       "      <td>4.100000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>10.00000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>44.883500</td>\n",
       "      <td>7.753784e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.490000e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         bathrooms      bedrooms      latitude    listing_id     longitude  \\\n",
       "count  49352.00000  49352.000000  49352.000000  4.935200e+04  49352.000000   \n",
       "mean       1.21218      1.541640     40.741545  7.024055e+06    -73.955716   \n",
       "std        0.50142      1.115018      0.638535  1.262746e+05      1.177912   \n",
       "min        0.00000      0.000000      0.000000  6.811957e+06   -118.271000   \n",
       "25%        1.00000      1.000000     40.728300  6.915888e+06    -73.991700   \n",
       "50%        1.00000      1.000000     40.751800  7.021070e+06    -73.977900   \n",
       "75%        1.00000      2.000000     40.774300  7.128733e+06    -73.954800   \n",
       "max       10.00000      8.000000     44.883500  7.753784e+06      0.000000   \n",
       "\n",
       "              price  \n",
       "count  4.935200e+04  \n",
       "mean   3.830174e+03  \n",
       "std    2.206687e+04  \n",
       "min    4.300000e+01  \n",
       "25%    2.500000e+03  \n",
       "50%    3.150000e+03  \n",
       "75%    4.100000e+03  \n",
       "max    4.490000e+06  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "测试集数据了解"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>building_id</th>\n",
       "      <th>created</th>\n",
       "      <th>description</th>\n",
       "      <th>display_address</th>\n",
       "      <th>features</th>\n",
       "      <th>latitude</th>\n",
       "      <th>listing_id</th>\n",
       "      <th>longitude</th>\n",
       "      <th>manager_id</th>\n",
       "      <th>photos</th>\n",
       "      <th>price</th>\n",
       "      <th>street_address</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>79780be1514f645d7e6be99a3de696c5</td>\n",
       "      <td>2016-06-11 05:29:41</td>\n",
       "      <td>Large with awesome terrace--accessible via bed...</td>\n",
       "      <td>Suffolk Street</td>\n",
       "      <td>[Elevator, Laundry in Building, Laundry in Uni...</td>\n",
       "      <td>40.7185</td>\n",
       "      <td>7142618</td>\n",
       "      <td>-73.9865</td>\n",
       "      <td>b1b1852c416d78d7765d746cb1b8921f</td>\n",
       "      <td>[https://photos.renthop.com/2/7142618_1c45a2c8...</td>\n",
       "      <td>2950</td>\n",
       "      <td>99 Suffolk Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2016-06-24 06:36:34</td>\n",
       "      <td>Prime Soho - between Bleecker and Houston - Ne...</td>\n",
       "      <td>Thompson Street</td>\n",
       "      <td>[Pre-War, Dogs Allowed, Cats Allowed]</td>\n",
       "      <td>40.7278</td>\n",
       "      <td>7210040</td>\n",
       "      <td>-74.0000</td>\n",
       "      <td>d0b5648017832b2427eeb9956d966a14</td>\n",
       "      <td>[https://photos.renthop.com/2/7210040_d824cc71...</td>\n",
       "      <td>2850</td>\n",
       "      <td>176 Thompson Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3dbbb69fd52e0d25131aa1cd459c87eb</td>\n",
       "      <td>2016-06-03 04:29:40</td>\n",
       "      <td>New York chic has reached a new level ...</td>\n",
       "      <td>101 East 10th Street</td>\n",
       "      <td>[Doorman, Elevator, No Fee]</td>\n",
       "      <td>40.7306</td>\n",
       "      <td>7103890</td>\n",
       "      <td>-73.9890</td>\n",
       "      <td>9ca6f3baa475c37a3b3521a394d65467</td>\n",
       "      <td>[https://photos.renthop.com/2/7103890_85b33077...</td>\n",
       "      <td>3758</td>\n",
       "      <td>101 East 10th Street</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>783d21d013a7e655bddc4ed0d461cc5e</td>\n",
       "      <td>2016-06-11 06:17:35</td>\n",
       "      <td>Step into this fantastic new Construction in t...</td>\n",
       "      <td>South Third Street\\r</td>\n",
       "      <td>[Roof Deck, Balcony, Elevator, Laundry in Buil...</td>\n",
       "      <td>40.7109</td>\n",
       "      <td>7143442</td>\n",
       "      <td>-73.9571</td>\n",
       "      <td>0b9d5db96db8472d7aeb67c67338c4d2</td>\n",
       "      <td>[https://photos.renthop.com/2/7143442_0879e9e0...</td>\n",
       "      <td>3300</td>\n",
       "      <td>251  South Third Street\\r</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100000</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>6134e7c4dd1a98d9aee36623c9872b49</td>\n",
       "      <td>2016-04-12 05:24:17</td>\n",
       "      <td>~Take a stroll in Central Park, enjoy the ente...</td>\n",
       "      <td>Midtown West, 8th Ave</td>\n",
       "      <td>[Common Outdoor Space, Cats Allowed, Dogs Allo...</td>\n",
       "      <td>40.7650</td>\n",
       "      <td>6860601</td>\n",
       "      <td>-73.9845</td>\n",
       "      <td>b5eda0eb31b042ce2124fd9e9fcfce2f</td>\n",
       "      <td>[https://photos.renthop.com/2/6860601_c96164d8...</td>\n",
       "      <td>4900</td>\n",
       "      <td>260 West 54th Street</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        bathrooms  bedrooms                       building_id  \\\n",
       "0             1.0         1  79780be1514f645d7e6be99a3de696c5   \n",
       "1             1.0         2                                 0   \n",
       "100           1.0         1  3dbbb69fd52e0d25131aa1cd459c87eb   \n",
       "1000          1.0         2  783d21d013a7e655bddc4ed0d461cc5e   \n",
       "100000        2.0         2  6134e7c4dd1a98d9aee36623c9872b49   \n",
       "\n",
       "                    created  \\\n",
       "0       2016-06-11 05:29:41   \n",
       "1       2016-06-24 06:36:34   \n",
       "100     2016-06-03 04:29:40   \n",
       "1000    2016-06-11 06:17:35   \n",
       "100000  2016-04-12 05:24:17   \n",
       "\n",
       "                                              description  \\\n",
       "0       Large with awesome terrace--accessible via bed...   \n",
       "1       Prime Soho - between Bleecker and Houston - Ne...   \n",
       "100             New York chic has reached a new level ...   \n",
       "1000    Step into this fantastic new Construction in t...   \n",
       "100000  ~Take a stroll in Central Park, enjoy the ente...   \n",
       "\n",
       "              display_address  \\\n",
       "0              Suffolk Street   \n",
       "1             Thompson Street   \n",
       "100      101 East 10th Street   \n",
       "1000     South Third Street\\r   \n",
       "100000  Midtown West, 8th Ave   \n",
       "\n",
       "                                                 features  latitude  \\\n",
       "0       [Elevator, Laundry in Building, Laundry in Uni...   40.7185   \n",
       "1                   [Pre-War, Dogs Allowed, Cats Allowed]   40.7278   \n",
       "100                           [Doorman, Elevator, No Fee]   40.7306   \n",
       "1000    [Roof Deck, Balcony, Elevator, Laundry in Buil...   40.7109   \n",
       "100000  [Common Outdoor Space, Cats Allowed, Dogs Allo...   40.7650   \n",
       "\n",
       "        listing_id  longitude                        manager_id  \\\n",
       "0          7142618   -73.9865  b1b1852c416d78d7765d746cb1b8921f   \n",
       "1          7210040   -74.0000  d0b5648017832b2427eeb9956d966a14   \n",
       "100        7103890   -73.9890  9ca6f3baa475c37a3b3521a394d65467   \n",
       "1000       7143442   -73.9571  0b9d5db96db8472d7aeb67c67338c4d2   \n",
       "100000     6860601   -73.9845  b5eda0eb31b042ce2124fd9e9fcfce2f   \n",
       "\n",
       "                                                   photos  price  \\\n",
       "0       [https://photos.renthop.com/2/7142618_1c45a2c8...   2950   \n",
       "1       [https://photos.renthop.com/2/7210040_d824cc71...   2850   \n",
       "100     [https://photos.renthop.com/2/7103890_85b33077...   3758   \n",
       "1000    [https://photos.renthop.com/2/7143442_0879e9e0...   3300   \n",
       "100000  [https://photos.renthop.com/2/6860601_c96164d8...   4900   \n",
       "\n",
       "                   street_address  \n",
       "0               99 Suffolk Street  \n",
       "1             176 Thompson Street  \n",
       "100          101 East 10th Street  \n",
       "1000    251  South Third Street\\r  \n",
       "100000       260 West 54th Street  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 74659 entries, 0 to 99999\n",
      "Data columns (total 14 columns):\n",
      "bathrooms          74659 non-null float64\n",
      "bedrooms           74659 non-null int64\n",
      "building_id        74659 non-null object\n",
      "created            74659 non-null object\n",
      "description        74659 non-null object\n",
      "display_address    74659 non-null object\n",
      "features           74659 non-null object\n",
      "latitude           74659 non-null float64\n",
      "listing_id         74659 non-null int64\n",
      "longitude          74659 non-null float64\n",
      "manager_id         74659 non-null object\n",
      "photos             74659 non-null object\n",
      "price              74659 non-null int64\n",
      "street_address     74659 non-null object\n",
      "dtypes: float64(3), int64(3), object(8)\n",
      "memory usage: 8.5+ MB\n"
     ]
    }
   ],
   "source": [
    "test.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>latitude</th>\n",
       "      <th>listing_id</th>\n",
       "      <th>longitude</th>\n",
       "      <th>price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>74659.000000</td>\n",
       "      <td>74659.000000</td>\n",
       "      <td>74659.000000</td>\n",
       "      <td>7.465900e+04</td>\n",
       "      <td>74659.000000</td>\n",
       "      <td>7.465900e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.212915</td>\n",
       "      <td>1.544663</td>\n",
       "      <td>40.735060</td>\n",
       "      <td>7.024001e+06</td>\n",
       "      <td>-73.945282</td>\n",
       "      <td>3.749033e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.649820</td>\n",
       "      <td>1.107014</td>\n",
       "      <td>0.806687</td>\n",
       "      <td>1.264496e+05</td>\n",
       "      <td>1.487795</td>\n",
       "      <td>9.713092e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6.811958e+06</td>\n",
       "      <td>-121.488000</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>40.727800</td>\n",
       "      <td>6.915516e+06</td>\n",
       "      <td>-73.991800</td>\n",
       "      <td>2.495000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>40.751600</td>\n",
       "      <td>7.021738e+06</td>\n",
       "      <td>-73.977700</td>\n",
       "      <td>3.150000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>40.774300</td>\n",
       "      <td>7.129166e+06</td>\n",
       "      <td>-73.954700</td>\n",
       "      <td>4.100000e+03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>112.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>42.872700</td>\n",
       "      <td>7.761779e+06</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.675000e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          bathrooms      bedrooms      latitude    listing_id     longitude  \\\n",
       "count  74659.000000  74659.000000  74659.000000  7.465900e+04  74659.000000   \n",
       "mean       1.212915      1.544663     40.735060  7.024001e+06    -73.945282   \n",
       "std        0.649820      1.107014      0.806687  1.264496e+05      1.487795   \n",
       "min        0.000000      0.000000      0.000000  6.811958e+06   -121.488000   \n",
       "25%        1.000000      1.000000     40.727800  6.915516e+06    -73.991800   \n",
       "50%        1.000000      1.000000     40.751600  7.021738e+06    -73.977700   \n",
       "75%        1.000000      2.000000     40.774300  7.129166e+06    -73.954700   \n",
       "max      112.000000      7.000000     42.872700  7.761779e+06      0.000000   \n",
       "\n",
       "              price  \n",
       "count  7.465900e+04  \n",
       "mean   3.749033e+03  \n",
       "std    9.713092e+03  \n",
       "min    1.000000e+00  \n",
       "25%    2.495000e+03  \n",
       "50%    3.150000e+03  \n",
       "75%    4.100000e+03  \n",
       "max    1.675000e+06  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "去掉ID列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.drop([\"listing_id\"], axis=1, inplace=True)\n",
    "test_id = test[\"listing_id\"]\n",
    "test.drop([\"listing_id\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "将类别编码为数字"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_map = {'low': 2, 'medium': 1, 'high': 0}\n",
    "train['interest_level'] = train['interest_level'].apply(lambda x: y_map[x])\n",
    "y_train = train['interest_level']\n",
    "train.drop(['interest_level'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "去掉噪声数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAEWCAYAAADPZygPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAGLtJREFUeJzt3XuQpXV95/H3xxkQVEZERkUGHYyj\nMibrrVXUVFR0dfA2lqUJaGR0MSTG666WQXc3kER3TcqSxA2aoBDHS0SCrqLBRQTU0lWkERURXSYK\nMkJkkNt4KRD87h/n18mx6enbzOnfTJ/3q+pUP8/3ufy+pxnO5zyXPidVhSRJPdyldwOSpPFlCEmS\nujGEJEndGEKSpG4MIUlSN4aQJKkbQ0iapySfT/KKHSz7TJJNIxr3lUl+nOSnSe49ijEWK8lTkmwd\nmr8syVM6tqQ9zMreDUijkORK4BVV9bmlGK+qjhzFfpPsBbwTOLyqvjmKMRbYTwHrqmrLTMur6uFL\n3JL2cB4JSbu3+wL7AJctdMMM7BH/jyfxDfGY2iP+gUq7UpI/SLIlyQ1Jzkpy/6FlT0xyUZKb288n\n7mAfByX5VpI3tvl/O1WX5GVJvpTkHUluTPKDJEcObXtoki8m2Z7kc0lOTvKhGcZ4CPC9NntTkvPn\n6rH18bYkXwZ+Djxohv0e1ta7qZ0+e9607V8xNP+yJF9q019s5W+2U4O/N8O+r0zy9DZ9lyTHJ/mX\nJD9JckaSA9qytUkqybFJfgicn2SfJB9q697Untt9Z/r9a/kwhDRWkhwB/E/gd4GDgKuA09uyA4B/\nBt4F3JvBabB/nn4dJsla4AvA31bVO3Yw1OMZBMiBwF8BpyZJW/aPwNfaGCcCL51pB1X1/4Cp01v7\nV9UR8+zxpcBxwH7t+Q33vhfwKeCzwH2A1wAfTvLQHTyP4X5+p00+oqruUVUfnWOT1wLPB54M3B+4\nETh52jpPBg4DnglsAu4JHNKe2x8Bv5irL+3ZDCGNm5cAp1XV16vqVuDNwBNasDwbuKKqPlhVt1fV\nR4DvAs8d2n498HnghKo6ZZZxrqqq91bVHcBmBoF33yQPAB4L/GlV3VZVXwLOWkD/8+nx/VV1WVv+\ny2nbHw7cA3h7G/984NPA0QvoYb7+EPivVbW1/a5PBF447dTbiVX1s6r6BfBLBuHz4Kq6o6ourqpb\nRtCXdiOGkMbN/Rk6OqiqnwI/AQ6evqy5qi2b8hLgR8CZc4zzr0Nj/LxN3qONccNQDeDqxfa/gx5n\n29/9gaur6lezbL+rPBD43+3U2k3A5cAdDK5zTRnu9YPAOcDpSa5J8lftyE3LmCGkcXMNgxdHAJLc\nncG77x9NX9Y8oC2bciJwPfCPSVYsYvxrgQOS3G2odsgCtp9Pj7N9NP41wCHTblgY3v5nwHBv91tA\nb9NdDRxZVfsPPfapqhl7rapfVtWfVdV64InAc4BjdmJ87QEMIS1ne7WL3VOPlQyux7w8ySOT3BX4\nH8CFVXUlcDbwkCQvTrKyXXhfz+B01ZRfAi8C7g58cKF3n1XVVcAkcGKSvZM8gV8/lTaX+fQ4mwsZ\nBM2bkuzV/qbnubTrYsA3gBckuVuSBwPHTtv+x8xws8MO/B3wtiQPBEiyOsnGHa2c5KlJfquF+y0M\nftd3zHMs7aEMIS1nZzO4sD31OLGqzgP+O/AxBkclvwEcBVBVP2Hw7vsNDE7RvQl4TlVdP7zTqroN\neAGDC/unLeI26JcAT2hjvBX4KHDrfDacb4+zbH8b8DzgSAZHdO8Gjqmq77ZVTgJuYxA2m4EPT9vF\nicDmdortd+cY7m8YXO/6bJLtwFcZ3LCxI/djcJrzFgan7r4A3OmuQS0v8UvtpL6SfBT4blWd0LsX\naal5JCQtsSSPTfIb7e9oNgAbgU/07kvqwb9Slpbe/YCPM7ghYivwyqq6pG9LUh+ejpMkdePpOElS\nN56Om8OBBx5Ya9eu7d2GJO1RLr744uuravVc6xlCc1i7di2Tk5O925CkPUqS6Z/sMSNPx0mSujGE\nJEndGEKSpG4MIUlSN4aQJKkbQ2gZWbUKkjs/Vq3q3ZkkzcwQWka2b19YXZJ6M4QkSd0YQpKkbgwh\nSVI3hpAkqRtDaBnZb7+F1SWpNz/AdBm55ZbeHUjSwngkJEnqxhCSJHVjCEmSujGEJEndGEKSpG4M\nIUlSN4aQJKkbQ0iS1I0hJEnqxhCSJHVjCEmSujGEJEndGEKSpG4MIUlSNyMPoSQrklyS5NNt/tAk\nFya5IslHk+zd6ndt81va8rVD+3hzq38vyTOH6htabUuS44fqCx5DkrT0luJI6HXA5UPzfwmcVFXr\ngBuBY1v9WODGqnowcFJbjyTrgaOAhwMbgHe3YFsBnAwcCawHjm7rLngMSVIfIw2hJGuAZwPva/MB\njgDObKtsBp7fpje2edryp7X1NwKnV9WtVfUDYAvwuPbYUlXfr6rbgNOBjYscQ5LUwaiPhP4aeBPw\nqzZ/b+Cmqrq9zW8FDm7TBwNXA7TlN7f1/60+bZsd1Rczxq9JclySySST27ZtW/izliTNy8hCKMlz\ngOuq6uLh8gyr1hzLdlV9rvH/vVB1SlVNVNXE6tWrZ9hEkrQrrBzhvp8EPC/Js4B9gFUMjoz2T7Ky\nHYmsAa5p628FDgG2JlkJ3BO4Yag+ZXibmerXL2IMSVIHIzsSqqo3V9WaqlrL4MaC86vqJcAFwAvb\napuAT7bps9o8bfn5VVWtflS7s+1QYB3wNeAiYF27E27vNsZZbZuFjiFJ6mCUR0I78ifA6UneClwC\nnNrqpwIfTLKFwdHJUQBVdVmSM4DvALcDr6qqOwCSvBo4B1gBnFZVly1mDElSH/FAYHYTExM1OTnZ\nuw1J2qMkubiqJuZaz09MkCR1YwhJkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6sYQkiR1\nYwhJkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6sYQkiR1YwhJkroxhCRJ3RhCkqRuDCFJ\nUjeGkCSpG0NIktSNISRJ6sYQkiR1YwhJkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6sYQ\nkiR1YwhJkroxhCRJ3YwshJLsk+RrSb6Z5LIkf9bqhya5MMkVST6aZO9Wv2ub39KWrx3a15tb/XtJ\nnjlU39BqW5IcP1Rf8BiSpKU3yiOhW4EjquoRwCOBDUkOB/4SOKmq1gE3Ase29Y8FbqyqBwMntfVI\nsh44Cng4sAF4d5IVSVYAJwNHAuuBo9u6LHQMSVIfIwuhGvhpm92rPQo4Ajiz1TcDz2/TG9s8bfnT\nkqTVT6+qW6vqB8AW4HHtsaWqvl9VtwGnAxvbNgsdQ5LUwUivCbUjlm8A1wHnAv8C3FRVt7dVtgIH\nt+mDgasB2vKbgXsP16dts6P6vRcxxvS+j0symWRy27Zti3vykqQ5jTSEquqOqnoksIbBkcthM63W\nfs50RFK7sD7bGL9eqDqlqiaqamL16tUzbCJJ2hWW5O64qroJ+DxwOLB/kpVt0Rrgmja9FTgEoC2/\nJ3DDcH3aNjuqX7+IMSRJHYzy7rjVSfZv0/sCTwcuBy4AXthW2wR8sk2f1eZpy8+vqmr1o9qdbYcC\n64CvARcB69qdcHszuHnhrLbNQseQJHWwcu5VFu0gYHO7i+0uwBlV9ekk3wFOT/JW4BLg1Lb+qcAH\nk2xhcHRyFEBVXZbkDOA7wO3Aq6rqDoAkrwbOAVYAp1XVZW1ff7KQMSRJfcQDgdlNTEzU5ORk7zYk\naY+S5OKqmphrPT8xQZLUjSEkSerGEJIkdWMISZK6MYQkSd0YQpKkbgwhSVI3hpAkqRtDSJLUjSEk\nSerGEJIkdWMISZK6MYQkSd0YQpKkbgwhSVI3hpAkqRtDSJLUzbxDKMkDkzy9Te+bZL/RtSVJGgfz\nCqEkfwCcCfx9K60BPjGqpiRJ42G+R0KvAp4E3AJQVVcA9xlVU5Kk8TDfELq1qm6bmkmyEqjRtCRJ\nGhfzDaEvJHkLsG+S/wj8E/Cp0bUlSRoH8w2h44FtwKXAHwJnA/9tVE1JksbDynmuty9wWlW9FyDJ\nilb7+agakyQtf/M9EjqPQehM2Rf43K5vR5I0TuYbQvtU1U+nZtr03UbTkiRpXMw3hH6W5NFTM0ke\nA/xiNC1JksbFfK8JvR74pyTXtPmDgN8bTUuSpHExrxCqqouSPAx4KBDgu1X1y5F2Jkla9mYNoSRH\nVNX5SV4wbdG6JFTVx0fYmyRpmZvrSOjJwPnAc2dYVoAhJElatFlDqKpOSHIX4DNVdcYS9SRJGhNz\n3h1XVb8CXr0EvUiSxsx8b9E+N8kbkxyS5ICpx0g7kyQte/O9Rfs/MbgG9MfT6g/ate1IksbJfI+E\n1gMnA98EvgH8L+Dhs23QjpouSHJ5ksuSvK7VD0hybpIr2s97tXqSvCvJliTfmvbHsZva+lck2TRU\nf0ySS9s270qSxY4hSVp68w2hzcBhwLsYBNBhrTab24E3VNVhwOHAq5KsZ/CJ3OdV1ToGn0l3fFv/\nSGBdexwHvAcGgQKcADweeBxwwlSotHWOG9puQ6svaAxJUh/zDaGHVtUrquqC9jiOwR+u7lBVXVtV\nX2/T24HLgYOBjfx7gG0Gnt+mNwIfqIGvAvsnOQh4JnBuVd1QVTcC5wIb2rJVVfWVqirgA9P2tZAx\nJEkdzDeELkly+NRMkscDX57vIEnWAo8CLgTuW1XXwiCo+PevCT8YuHpos62tNlt96wx1FjHG9H6P\nSzKZZHLbtm3zfZqSpAWabwg9Hvi/Sa5MciXwFeDJ7XrMt2bbMMk9gI8Br6+qW2ZbdYZaLaI+azvz\n2aaqTqmqiaqaWL169Ry7lCQt1nzvjtsw9yp3lmQvBgH04aGP+PlxkoOq6tp2Kuy6Vt8KHDK0+Rrg\nmlZ/yrT651t9zQzrL2YMSVIH8zoSqqqrZnvMtE27U+1U4PKqeufQorOAqTvcNgGfHKof0+5gOxy4\nuZ1KOwd4RpJ7tRsSngGc05ZtT3J4G+uYaftayBiSpA7meyS0GE8CXgpcmuQbrfYW4O3AGUmOBX4I\nvKgtOxt4FrCFwdeGvxygqm5I8hfARW29P6+qG9r0K4H3M/im18+0BwsdQ5LURwY3lmlHJiYmanJy\nsncbkrRHSXJxVU3Mtd58b0yQJGmXM4QkSd0YQpKkbgwhSVI3hpAkqRtDSJLUjSEkSerGEJIkdWMI\nSZK6MYQkSd0YQpKkbgwhSVI3hpAkqRtDSJLUjSEkSerGEJIkdWMISZK6MYQkSd0YQpKkbgwhSVI3\nhpAkqRtDSJLUjSEkSerGEJIkdWMISZK6MYQkSd0YQpKkbgwhSVI3htAysmoVJHd+rFrVuzNJmpkh\ntIxs376wuiT1ZghJkroxhCRJ3RhCkqRuDCFJUjeGkCSpm5GFUJLTklyX5NtDtQOSnJvkivbzXq2e\nJO9KsiXJt5I8emibTW39K5JsGqo/JsmlbZt3Jclix5Ak9THKI6H3Axum1Y4HzquqdcB5bR7gSGBd\nexwHvAcGgQKcADweeBxwwlSotHWOG9puw2LGkCT1M7IQqqovAjdMK28ENrfpzcDzh+ofqIGvAvsn\nOQh4JnBuVd1QVTcC5wIb2rJVVfWVqirgA9P2tZAxJEmdLPU1oftW1bUA7ed9Wv1g4Oqh9ba22mz1\nrTPUFzPGnSQ5Lslkkslt27Yt6AlKkuZvd7kxITPUahH1xYxx52LVKVU1UVUTq1evnmO3u4+qHT8k\naXe01CH046lTYO3nda2+FThkaL01wDVz1NfMUF/MGJKkTpY6hM4Cpu5w2wR8cqh+TLuD7XDg5nYq\n7RzgGUnu1W5IeAZwTlu2Pcnh7a64Y6btayFjSJI6WTmqHSf5CPAU4MAkWxnc5fZ24IwkxwI/BF7U\nVj8beBawBfg58HKAqrohyV8AF7X1/ryqpm52eCWDO/D2BT7THix0DElSPykvGMxqYmKiJicne7ch\nSXuUJBdX1cRc6+0uNyZIksaQISRJ6sYQkiR1YwhJkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NIktSN\nISRJ6sYQkiR1YwhJkroxhCRJ3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6sYQkiR1YwhJkroxhCRJ\n3RhCkqRuDCFJUjeGkCSpG0NIktSNISRJ6sYQkiR1YwhJkroxhCRJ3RhCI7BqFSR3fqxa1bszSdq9\nGEIjsH37wuq7iuEnaU9jCC0jvcKvp5lCd+qxHPV8o+GbHI2CISQtQq8X5J5vNDzC1ygYQtppO/si\n0etFZmfG3ZkX5HE7ettZ43iEP04MIQE798K4sy8SvV5kxvHFrVcAejSjHRm7EEqyIcn3kmxJcnzv\nfqRxMI6Bvyfq8WZhrEIoyQrgZOBIYD1wdJL1fbuSpN1DjzcLYxVCwOOALVX1/aq6DTgd2Ni5J0ka\nW+MWQgcDVw/Nb221X5PkuCSTSSa3bdu2ZM1JurP99ltYXXuWcQuhmS6/1p0KVadU1URVTaxevXoJ\n2pK0I7fcAlV3ftxyS+/OtCuMWwhtBQ4Zml8DXNOpF2lseDSjHRm3ELoIWJfk0CR7A0cBZ3XuSWNm\nZ16QZzoimHrMx85svzPbejSzZ+jxZmHl6Ha9+6mq25O8GjgHWAGcVlWX7fpxdvUeRz9ur217jt1r\nW194tbvq8W9zrEIIoKrOBs7u3YckafxOx0mSdiOGkCSpG0NIktSNISRJ6ibV61auPUSSbcBVO7GL\nA4Hrd1E7e4Jxe77gcx4X4/acd/b5PrCq5vxrf0NoxJJMVtVE7z6Wyrg9X/A5j4txe85L9Xw9HSdJ\n6sYQkiR1YwiN3im9G1hi4/Z8wec8LsbtOS/J8/WakCSpG4+EJEndGEKSpG4MoRFJsiHJ95JsSXJ8\n735GLckhSS5IcnmSy5K8rndPSyHJiiSXJPl0716WQpL9k5yZ5Lvtv/UTevc0akn+c/s3/e0kH0my\nT++edrUkpyW5Lsm3h2oHJDk3yRXt571GMbYhNAJJVgAnA0cC64Gjk6zv29XI3Q68oaoOAw4HXjUG\nzxngdcDlvZtYQn8D/J+qehjwCJb5c09yMPBaYKKqfpPBV8Ac1berkXg/sGFa7XjgvKpaB5zX5nc5\nQ2g0HgdsqarvV9VtwOnAxs49jVRVXVtVX2/T2xm8OB3ct6vRSrIGeDbwvt69LIUkq4DfAU4FqKrb\nquqmvl0tiZXAvklWAndjGX4bc1V9EbhhWnkjsLlNbwaeP4qxDaHROBi4emh+K8v8BXlYkrXAo4AL\n+3Yycn8NvAn4Ve9GlsiDgG3AP7RTkO9LcvfeTY1SVf0IeAfwQ+Ba4Oaq+mzfrpbMfavqWhi8yQTu\nM4pBDKHRyAy1sbgXPsk9gI8Br6+qZfsdokmeA1xXVRf37mUJrQQeDbynqh4F/IwRnaLZXbTrIBuB\nQ4H7A3dP8vt9u1peDKHR2AocMjS/hmV4CD9dkr0YBNCHq+rjvfsZsScBz0tyJYPTrUck+VDflkZu\nK7C1qqaOcM9kEErL2dOBH1TVtqr6JfBx4Imde1oqP05yEED7ed0oBjGERuMiYF2SQ5PszeBC5lmd\nexqpJGFwreDyqnpn735GrareXFVrqmotg/++51fVsn6HXFX/Clyd5KGt9DTgOx1bWgo/BA5Pcrf2\nb/xpLPObMYacBWxq05uAT45ikJWj2Om4q6rbk7waOIfB3TSnVdVlndsatScBLwUuTfKNVntLVZ3d\nsSfteq8BPtzeXH0feHnnfkaqqi5McibwdQZ3gF7CMvz4niQfAZ4CHJhkK3AC8HbgjCTHMgjjF41k\nbD+2R5LUi6fjJEndGEKSpG4MIUlSN4aQJKkbQ0iS1I0hJI1AkrXDn0g8j/VfluT+Q/NXJjlwNN1J\nuw9DSNo9vIzBx8LMW/tATWmPZghJo7MyyeYk32rfwXO3JH+a5KL23TSnZOCFwASDPwL9RpJ92/av\nSfL1JJcmeRhAkhPbdp8FPpBknyT/0Na5JMlT23o7qr8sySeSfCrJD5K8Osl/aet8NckBbb3XJvlO\n6/30pf/VaVwYQtLoPBQ4par+A3AL8MfA31bVY9t30+wLPKeqzgQmgZdU1SOr6hdt++ur6tHAe4A3\nDu33McDGqnox8CqAqvot4Ghgc/vStR3VAX4TeDGDrxx5G/Dz9oGkXwGOaescDzyq9f5Hu/S3Ig0x\nhKTRubqqvtymPwT8NvDUJBcmuRQ4Anj4LNtPfQjsxcDaofpZQ0H128AHAarqu8BVwENmqQNcUFXb\nq2obcDPwqVa/dGicbzE4Mvt9Bh9XI42EISSNzvTPxCrg3cAL2xHKe4HZvir61vbzDn79cx5/NjQ9\n09eGzFYf3i8Mvgvp1qHpqXGezeDbgR8DXOz1J42KISSNzgOSPKFNHw18qU1f37536YVD624H9lvE\nGF8EXgKQ5CHAA4DvzVKfU5K7AIdU1QUMvrRvf+Aei+hNmpPvbqTRuRzYlOTvgSsYXNu5F4PTXlcy\n+MqPKe8H/i7JL4AnMH/vbttdyuC02cuq6tYkO6rPZ58rgA8luSeDI6qTxuRrvNWBn6ItSerG03GS\npG4MIUlSN4aQJKkbQ0iS1I0hJEnqxhCSJHVjCEmSuvn/xwpSrZfXUgEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1b3aca90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(train.bathrooms, train.price, c = \"blue\", marker = \"s\")\n",
    "plt.title(\"Looking for outliers\")\n",
    "plt.xlabel(\"bathrooms\")\n",
    "plt.ylabel(\"price\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def remove_noise(df):\n",
    "    df= df[df.price < 10000]\n",
    "    df.loc[df[\"bathrooms\"] == 112, \"bathrooms\"] = 1.5\n",
    "    df.loc[df[\"bathrooms\"] == 10, \"bathrooms\"] = 1\n",
    "    df.loc[df[\"bathrooms\"] == 20, \"bathrooms\"] = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "构造新特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_price_room(df):\n",
    "    df['price_bathrooms'] =  (df[\"price\"])/ (df[\"bathrooms\"] +1.0)\n",
    "    df['price_bedrooms'] =  (df[\"price\"])/ (df[\"bedrooms\"] +1.0)\n",
    "    \n",
    "def create_room_diff_sum(df):\n",
    "    df[\"room_diff\"] = df[\"bathrooms\"] - df[\"bedrooms\"]\n",
    "    df[\"room_num\"] = df[\"bedrooms\"] + df[\"bathrooms\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "创建日期特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_created_date(df):\n",
    "    df['Date'] = pd.to_datetime(df['created'])\n",
    "    df['Year'] = df['Date'].dt.year\n",
    "    df['Month'] = df['Date'].dt.month\n",
    "    df['Day'] = df['Date'].dt.day\n",
    "    df['Wday'] = df['Date'].dt.dayofweek\n",
    "    df['Yday'] = df['Date'].dt.dayofyear\n",
    "    df['hour'] = df['Date'].dt.hour\n",
    "    df.drop(['Date', 'created'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "去掉描述特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_description(df):\n",
    "    df.drop(['description'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "将manager特征分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_manager_id(df):\n",
    "    managers_count = df['manager_id'].value_counts()\n",
    "\n",
    "    df['top_10_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[\n",
    "        managers_count.values >= np.percentile(managers_count.values, 90)] else 0)\n",
    "    df['top_50_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[\n",
    "        managers_count.values >= np.percentile(managers_count.values, 50)] else 0)\n",
    "    df['top_20_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[\n",
    "        managers_count.values >= np.percentile(managers_count.values, 80)] else 0)\n",
    "    df['top_30_manager'] = df['manager_id'].apply(lambda x: 1 if x in managers_count.index.values[\n",
    "        managers_count.values >= np.percentile(managers_count.values, 70)] else 0)\n",
    "    \n",
    "    df.drop(['manager_id'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "去掉building ID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_building_id(df):\n",
    "    df.drop(['building_id'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "处理图片特征，增加图片数量特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_photos(df):\n",
    "    df['photos_count'] = df['photos'].apply(lambda x: len(x))\n",
    "    df.drop(['photos'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "坐标特征聚类降维编码，K-means聚类算法，与中心的距离，并删除原来坐标特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_location_train(df):   \n",
    "    train_location = df.loc[:,[ 'latitude', 'longitude']]\n",
    "    \n",
    "    kmeans_cluster = KMeans(n_clusters=20)\n",
    "    res = kmeans_cluster.fit(train_location)\n",
    "    res = kmeans_cluster.predict(train_location)\n",
    "\n",
    "    df['cenroid'] = res\n",
    "\n",
    "    center = [ train_location['latitude'].mean(), train_location['longitude'].mean()]\n",
    "    df['distance'] = abs(df['latitude'] - center[0]) + abs(df['longitude'] - center[1])\n",
    "    \n",
    "    df.drop(['latitude', 'longitude'], axis=1, inplace=True)\n",
    "    \n",
    "    return kmeans_cluster,center"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_location_test(df, kmeans_cluster, center):   \n",
    "    test_location = df.loc[:,[ 'latitude', 'longitude']]\n",
    "    \n",
    "    res = kmeans_cluster.predict(test_location)\n",
    "\n",
    "    df['cenroid'] = res\n",
    "\n",
    "    df['distance'] = abs(df['latitude'] - center[0]) + abs(df['longitude'] - center[1])\n",
    "    df.drop(['latitude', 'longitude'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "直接删除地址特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_display_address(df):\n",
    "    df.drop(['display_address'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "删除街道地址特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_street_address(df):\n",
    "    df = df.drop(['street_address'], axis=1,inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "描述特征处理，\n",
    "特征中单词的词频，相当于以数据集features中出现的词语为字典的one-hot编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_features_train_test(df_train, df_test):\n",
    "    n_train_samples = len(df_train.index)\n",
    "    \n",
    "    df_train_test = pd.concat((df_train, df_test), axis=0)\n",
    "    df_train_test['features2'] = df_train_test['features']\n",
    "    df_train_test['features2'] = df_train_test['features2'].apply(lambda x: ' '.join(x))\n",
    "\n",
    "    c_vect = CountVectorizer(stop_words='english', max_features=200, ngram_range=(1, 1), decode_error='ignore')\n",
    "    c_vect_sparse = c_vect.fit_transform(df_train_test['features2'])\n",
    "    c_vect_sparse_cols = c_vect.get_feature_names()\n",
    "\n",
    "    df_train.drop(['features'], axis=1, inplace=True)\n",
    "    df_test.drop(['features'], axis=1, inplace=True)\n",
    "    \n",
    "    #hstack作为特征处理的最后一步，先将其他所有特征都转换成数值型特征才能处理,稀疏表示\n",
    "    df_train_sparse = sparse.hstack([df_train, c_vect_sparse[:n_train_samples,:]]).tocsr()\n",
    "    df_test_sparse = sparse.hstack([df_test, c_vect_sparse[n_train_samples:,:]]).tocsr()\n",
    "    \n",
    "    #常规datafrmae\n",
    "    tmp = pd.DataFrame(c_vect_sparse.toarray()[:n_train_samples,:],columns = c_vect_sparse_cols, index=df_train.index)\n",
    "    df_train = pd.concat([df_train, tmp], axis=1)\n",
    "    \n",
    "    tmp = pd.DataFrame(c_vect_sparse.toarray()[n_train_samples:,:],columns = c_vect_sparse_cols, index=df_test.index)\n",
    "    df_test = pd.concat([df_test, tmp], axis=1)\n",
    "    \n",
    "    #df_test = pd.concat([df_test, tmp[n_train_samples:,:]], axis=1)\n",
    "  \n",
    "    return df_train_sparse,df_test_sparse,df_train, df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_features_test(df, c_vect):\n",
    "    df['features2'] = df['features']\n",
    "    df['features2'] = df['features2'].apply(lambda x: ' '.join(x))\n",
    "\n",
    "    c_vect_sparse = c_vect.transform(df['features2'])\n",
    "    c_vect_sparse_cols = c_vect.get_feature_names()\n",
    "\n",
    "    df.drop(['features', 'features2'], axis=1, inplace=True)\n",
    "    \n",
    "    #hstack作为特征处理的最后一步，先将其他所有特征都转换成数值型特征才能处理\n",
    "    df_sparse = sparse.hstack([df, c_vect_sparse]).tocsr()\n",
    "    \n",
    "    tmp = pd.DataFrame(c_vect_sparse.toarray(),columns = c_vect_sparse_cols, index=df.index)\n",
    "    df = pd.concat([df, tmp], axis=1)\n",
    "    \n",
    "    return df_sparse, df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "训练样本特征处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:517: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    }
   ],
   "source": [
    "remove_noise(train)\n",
    "\n",
    "create_price_room(train)\n",
    "create_room_diff_sum(train)\n",
    "\n",
    "process_created_date(train)\n",
    "\n",
    "process_description(train)\n",
    "\n",
    "process_manager_id(train)\n",
    "\n",
    "process_building_id(train)\n",
    "process_photos(train)\n",
    "\n",
    "kmeans_cluster,center = process_location_train(train)\n",
    "process_street_address(train)\n",
    "process_display_address(train)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "测试样本特征处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:517: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    }
   ],
   "source": [
    "remove_noise(test)\n",
    "\n",
    "create_price_room(test)\n",
    "create_room_diff_sum(test)\n",
    "\n",
    "process_created_date(test)\n",
    "\n",
    "process_description(test)\n",
    "\n",
    "process_manager_id(test)\n",
    "\n",
    "process_building_id(test)\n",
    "process_photos(test)\n",
    "\n",
    "process_location_test(test, kmeans_cluster, center)\n",
    "\n",
    "process_street_address(test)\n",
    "process_display_address(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "特征描述字段词频处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_sparse,X_test_sparse,train,test = process_features_train_test(train,test)\n",
    "\n",
    "train = pd.concat([train, y_train], axis=1)\n",
    "test= pd.concat([test_id,test],axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "保存为csv文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train.to_csv('./data/RentListingInquries_FE_train.csv', index=False)\n",
    "test.to_csv('./data/RentListingInquries_FE_test.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
